"""
机器学习第一节：
    1、生成dense vetcor数据
    2、产生spars vector的三种方法
    3、普通文件转换为DenseVector的方法
"""

from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession

import numpy as np
# 要注意LabelPoint和Vector 来自不同的物件
from pyspark.mllib.linalg import Vectors
# regression(回归)
from pyspark.mllib.regression import LabeledPoint

spark = SparkSession.builder.master("local").appName("test").getOrCreate()
sc = spark.sparkContext

# 生成dense vetcor数据
x = [1, 2, 3, 4, 5]
dense_x = Vectors.dense(x)
print("dense_x = " + str(dense_x))
"""
dense vetcor
type(dense_x)
Out[10]: pyspark.mllib.linalg.DenseVector
"""

# 产生spars vector的三种方法
sparse_x = Vectors.sparse(5, {1: 1.0, 3: 5.5})
print("sparse_x = " + str(sparse_x))
sparse_y = Vectors.sparse(5, [(1, 1.0), (3, 5.5)])
print("sparse_y = " + str(sparse_y))
sparse_z = Vectors.sparse(5, [1, 3], [1.0, 5.5])
print("sparse_z = {}".format(str(sparse_z)))
"""
这三种方法都是生成spars vector的方法
sparse_x = (5,[1,3],[1.0,5.5])
sparse_y = (5,[1,3],[1.0,5.5])
sparse_z = (5,[1,3],[1.0,5.5])
"""

# 确认sparse vector
print("确认sparse vector{}".format(sparse_x.toArray()))
"""
#执行结果
确认sparse vector[0.  1.  0.  5.5 0. ]
"""


# 提取sparse
def print_sparse(x):
    for i in range(x.size):
        # 当saprse vector最后一位遇到缺值会因为出现异常省略出现Index Error
        try:
            print(x[i])
        except IndexError:
            print(0.0)


# 如下调用返回源数据
print_sparse(sparse_x)
"""
执行结果
0.0
1.0
0.0
5.5
0.0
#如果直接提取sparse_x数据
sparse_x[1]
1.0
"""

# labelPoint
data_label = [
    LabeledPoint(0.0, [0.0, 1.0, 1.0]),
    LabeledPoint(1.0, [1.0, 1.0, 2.0]),
    LabeledPoint(1.0, [2.0, 3.0, 2.0]),
    LabeledPoint(0.0, [3.0, 2.0, 5.0]),
]

# 将原始资料转换成Vectors
data = spark.read.csv("pyspark_data/ratings.csv", header=True)
# 拿出来一点资料
sample_data = data.sample(False, 0.9, 1)
# 在转换的时候会dataframe不能直接转换成sparse vector，需要先转换成rdd
dense_data = sample_data.rdd.map(lambda x: Vectors.dense(x))
dense_data.take(5)
"""
# type(dense_data)
# Out[32]: pyspark.rdd.PipelinedRDD
[DenseVector([3.0, 7155.0, 3.5, 1164885564.0]),
 DenseVector([3.0, 33750.0, 3.5, 1164887688.0]),
 DenseVector([4.0, 153.0, 5.0, 844416699.0]),
 DenseVector([4.0, 349.0, 3.0, 844416699.0])]
 
"""

